#coding:utf-8
from scrapy import Request
from scrapy.spiders import Spider
from douban_movies.items import DoubanMoviesItem
import json

class DoubanSpider(Spider):
    name = "movies"

    def __init__(self):
        self.page_start = 0

    def start_requests(self):
        url = "https://movie.douban.com/j/search_subjects?type=movie&tag=华语&sort=recommend&page_limit=20&page_start=0"
        yield Request(url)

    def parse(self, response):
        json_text = response.text
        movies_dict = json.loads(json_text)
        if not movies_dict["subjects"]:
            return

        for one_movie in movies_dict["subjects"]:
            movie_url = one_movie["url"][:-1]
            movie_url = movie_url.replace("subject/", "j/subject_abstract?subject_id=")
            yield Request(movie_url, callback = self.movie_parse)

        #获取下一页
        self.page_start += 20
        next_url = "https://movie.douban.com/j/search_subjects?type=movie&tag=华语&sort=recommend&page_limit=20&page_start=%d" % self.page_start
        yield Request(next_url)

    def movie_parse(self, response):
        json_text = response.text
        movie_dict = json.loads(json_text)
        item = DoubanMoviesItem()
        item["title"] = movie_dict["subject"]["title"]
        directors_dict = movie_dict["subject"]["directors"]
        item["directors"] = ",".join(directors_dict)
        actors_list = movie_dict["subject"]["actors"]
        item["actors"] = ",".join(actors_list)
        item["rate"] = movie_dict["subject"]["rate"]

        yield item



        
        
